package com.xiaotu.spider.license.pageprocessor;

import com.xiaotu.common.model.TVShootingPlanModel;
import com.xiaotu.common.util.CacheHandler;
import com.xiaotu.common.util.Constants;
import com.xiaotu.common.util.RegexUtils;
import com.xiaotu.spider.AUrlPageProcessor;
import org.apache.poi.ss.usermodel.Cell;
import org.apache.poi.ss.usermodel.Row;
import org.apache.poi.ss.usermodel.Sheet;
import org.apache.poi.ss.usermodel.Workbook;
import org.apache.poi.xssf.usermodel.XSSFWorkbook;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import us.codecraft.webmagic.Page;
import us.codecraft.webmagic.selector.Html;

import java.io.BufferedInputStream;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.HttpURLConnection;
import java.net.URL;
import java.nio.charset.Charset;
import java.util.*;
import java.util.zip.ZipEntry;
import java.util.zip.ZipInputStream;

/**
 * 广电总局全国重点网络影视剧拍摄规划登记备案情况
 *
 * @author 张晓
 * @create 2020-06-30 16:35
 */
@Component("TVShootPlanPageProcessor")
public class TVShootPlanPageProcessor extends AUrlPageProcessor {

    private static final String PREFIX = "http://211.146.10.138:8080/YSJBA/广电总局";
    private static final String SUFFIX = "全国重点网络影视剧拍摄规划登记备案情况.zip";
    private static final String REQUSET_URL = "http://211.146.10.138:8080/YSJBA/";
    private static final String REGEX_STR = "[0-9]{4}年[0-9]{1,2}月";
    private List<Map<String, String>> listAll = new ArrayList<Map<String, String>>();


    InputStream is = null;
    ZipInputStream zin = null;
    BufferedInputStream bs = null;
    InputStream bis = null;


    @Autowired
    private CacheHandler cacheHandler;

    @Override
    protected String[] getStartRequestUrls() {
        return new String[]{REQUSET_URL};
    }

    @Override
    protected String getSpiderType() {
        return Constants.SpiderTaskType.TV_SHOOT_PLAN_URL;
    }

    @Override
    public String getPageDataType(Page page) {
        return Constants.PageDataType.SEARCH_LIST;
    }

    @Override
    public void process(Page page) {
        //获取下载压缩包路径进行匹配是否变更
        Html html = page.getHtml();
        List<String> aTagList = html.xpath("//div[@class='login_details']//div[@class='login_guide']//a/@href").all();
        for (String s : aTagList) {
            if (s.contains(SUFFIX)) {
                String currDate = getDate(s, PREFIX, SUFFIX);
                if (RegexUtils.regexFind(REGEX_STR, currDate)) {
                    if (!cacheHandler.cacheExist(Constants.SpiderTaskType.TV_SHOOT_PLAN_URL)) {
                        cacheHandler.hSet(Constants.SpiderTaskType.TV_SHOOT_PLAN_URL, currDate, s);
                        try {
                            processData(s, currDate, page);
                        } catch (Exception e) {
                            cacheHandler.hDel(Constants.SpiderTaskType.TV_SHOOT_PLAN_URL, currDate);
                            e.printStackTrace();
                        } finally {
                            closeResources(is, zin, bs, bis);
                        }
                    } else {
                        //校验是否进行爬取
                        Map<String, Object> map = cacheHandler.hGetAll(Constants.SpiderTaskType.TV_SHOOT_PLAN_URL, String.class);
                        List<Map.Entry<String, Object>> list = new ArrayList<Map.Entry<String, Object>>(map.entrySet());
                        Collections.sort(list, new Comparator<Map.Entry<String, Object>>() {
                            @Override
                            public int compare(Map.Entry<String, Object> o1, Map.Entry<String, Object> o2) {
                                return -o1.getKey().compareTo(o2.getKey());
                            }
                        });
                        //获取最大的
                        Map.Entry<String, Object> maxDateEntry = list.get(0);
                        String maxDate = maxDateEntry.getKey();
                        //将当前获取到的日期与缓存中最大的日期进行比较(猜想到有可能月份不连续)
                        //先按这样处理
                        if (currDate.compareTo(maxDate) >= 1) {
                            try {
                                cacheHandler.hSet(Constants.SpiderTaskType.TV_SHOOT_PLAN_URL, currDate, s);
                                processData(s, currDate, page);
                            } catch (Exception e) {
                                cacheHandler.hDel(Constants.SpiderTaskType.TV_SHOOT_PLAN_URL, currDate);
                                e.printStackTrace();
                            } finally {
                                closeResources(is, zin, bs, bis);
                            }
                        }
                    }
                }
            }
        }
    }


    //抓取数据
    public void processData(String destUrl, String planDate, Page page) throws IOException {
        URL url = new URL(destUrl);
        HttpURLConnection conn = (HttpURLConnection) url.openConnection();
        //设置超时间为3秒
        conn.setConnectTimeout(10 * 1000);
        //得到输入流
        is = conn.getInputStream();
        zin = new ZipInputStream(is, Charset.forName("utf-8"));
        bs = new BufferedInputStream(zin);
        ZipEntry ze;
        List<Map<String, String>> list = null;

        while ((ze = zin.getNextEntry()) != null && ze.getName().lastIndexOf(".xlsx") != -1) {
            //读取每个文件的字节，并放进数组
            byte[] bytes = new byte[(int) ze.getSize()];
            bs.read(bytes, 0, (int) ze.getSize());
            //将文件转换成流
            bis = new ByteArrayInputStream(bytes);
            Workbook wb = new XSSFWorkbook(bis);
            //获取sheet的个数
            int sheetCount = wb.getNumberOfSheets();
            for (int i = 0; i < sheetCount; i++) {
                list = new ArrayList<Map<String, String>>();
                Sheet sheet = wb.getSheetAt(i);
                //表格名字
                String sheetName = sheet.getSheetName();
                int firstRowIndex = sheet.getFirstRowNum();
                int lastRowIndex = sheet.getLastRowNum();
                int sheetNum = sheet.getRow(i).getPhysicalNumberOfCells();
                for (int rIndex = firstRowIndex; rIndex <= lastRowIndex; rIndex++) {
                    Row row = sheet.getRow(rIndex);
                    if (row != null) {
                        int firstCellIndex = row.getFirstCellNum();
                        Map<String, String> map = new HashMap<String, String>();
                        for (int cIndex = firstCellIndex; cIndex < sheetNum; cIndex++) {
                            Cell cell = row.getCell(cIndex);
                            String value = "";
                            if (cell != null) {
                                value = cell.toString().trim();
                                map.put("" + cIndex + "", value);
                            }
                        }
                        list.add(map);
                    }

                }
                formartData(sheetName, list);
                listAll.addAll(list);
            }
        }
        //封装数据
        decorateData(page, planDate);
    }


    //将数据进行封装
    public void decorateData(Page page, String planDate) {
        List<TVShootingPlanModel> dataList = new ArrayList<TVShootingPlanModel>();
        listAll.forEach(s -> {
            String tvname = null;
            String sets = null;
            String epoch = null;
            String subject = null;
            String applicant = null;
            String tvtype = null;
            //剧名
            tvname = s.get("tvname") == null ? null : s.get("tvname").toString();
            //集数
            sets = (s.get("sets") == null || s.get("sets").toString().equals("")) ? null : (s.get("sets").toString().replace(".0",""));
            //年代
            epoch = s.get("epoch") == null ? null : s.get("epoch").toString();
            //题材
            subject = s.get("subject") == null ? null : s.get("subject").toString();
            //申报机构
            applicant = s.get("applicant") == null ? null : s.get("applicant").toString();
            //剧类型
            tvtype = s.get("tvtype") == null ? null : s.get("tvtype").toString();

            dataList.add(new TVShootingPlanModel(
                    tvname,
                    sets,
                    epoch,
                    subject,
                    applicant,
                    tvtype,
                    planDate
            ));

        });

        //保存数据
        page.putField(TVShootingPlanModel.class.getName(), dataList);

    }


    //统一关闭流的资源
    public void closeResources(InputStream is, ZipInputStream zin, BufferedInputStream bs, InputStream byteArrayInputStream) {
        if (byteArrayInputStream != null) {
            try {
                byteArrayInputStream.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        if (bs != null) {
            try {
                bs.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

        if (zin != null) {
            try {
                zin.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        if (is != null) {
            try {
                is.close();
            } catch (IOException e) {
                e.printStackTrace();
            }
        }

    }


    //格式化数据
    public void formartData(String sheetName, List<Map<String, String>> sheetData) {

        //标题
        Map<String, String> titleMap = sheetData.get(1);
        for (int i = 2; i < sheetData.size(); i++) {
            Map<String, String> map = sheetData.get(i);
            Iterator<Map.Entry<String, String>> iterator = titleMap.entrySet().iterator();
            while (iterator.hasNext()) {
                Map.Entry<String, String> entry = iterator.next();
                map.put(getColumName(entry.getValue()), map.get((entry.getKey())));
                //将之前的key进行remove
                map.remove(entry.getKey());
            }
            //设置题材
            if (sheetName.contains("重大题材")) {
                map.put("subject", sheetName);
            } else {
                map.put("tvtype", sheetName);
            }
        }

        sheetData.remove(0);
        sheetData.remove(0);
    }

    public String getColumName(String key) {
        if (key.contains("序号")) {
            return "ordernum";
        } else if (key.contains("节目名称")) {
            return "tvname";
        } else if (key.contains("节目类型")) {
            return "tvtype";
        } else if (key.contains("集数")) {
            return "sets";
        } else if (key.contains("年代")) {
            return "epoch";
        } else if (key.contains("题材")) {
            return "subject";
        } else if (key.contains("申报单位") || key.contains("申报机构")) {
            return "applicant";
        }
        return null;
    }

    //截取日期
    public String getDate(String all, String begin, String end) {
        int startIndex = all.indexOf(begin) + begin.length();
        int endIndex = all.indexOf(end);
        String result = all.substring(startIndex, endIndex);
        return result;
    }

}
